"""
    机器学习算法：LogisticRegression 逻辑回归
        logistic regression（非线性逻辑回归）
            问题
                游戏：有输有赢
                sales：
    f-measure算法
"""
from __future__ import print_function, division
from pyspark.ml.classification import LogisticRegression
from pyspark.sql import SparkSession

import seaborn as sns
import pandas as pd

import time
import os
import csv
from numpy import array

spark = SparkSession.builder.master("local").appName("test").enableHiveSupport().getOrCreate()
sc = spark.sparkContext
# 读取数据
training = spark.read.format("libsvm").load("E:\\Python\\pyspark_demo01\\pyspark_data\\sample_libsvm_data.txt")

lr = LogisticRegression(maxIter=10, regParam=0.3, elasticNetParam=0.8)
lrModel = lr.fit(training)
trainingSummary = lrModel.summary
print("查看roc系数：{}".format(trainingSummary.roc))

objectiveHistory = trainingSummary.objectiveHistory
print("执行历史：{}".format(objectiveHistory))

# 评价模型系数
trainingSummary.roc.show()
print("areaunderRoc:{}".format(trainingSummary.areaUnderROC))

roc_df = trainingSummary.roc.toPandas()
roc_df.head()
# 画图
import matplotlib.pyplot as plt

plt.title('Receiver Operating Characteristic')
plt.plot(roc_df.FPR, roc_df.TPR, "b")
plt.legend(loc="lower right")
plt.ylabel("TPR")
plt.xlabel("FPR")
plt.show()

fMeasure = trainingSummary.fMeasureByThreshold
fMeasure.show()

fMeasure = trainingSummary.fMeasureByThreshold
maxFMeasure = fMeasure.groupBy().max("F-Measure").select('max(F-Measure)').head()
# 找最佳化的切点
bestThreshold = (fMeasure.where(fMeasure['F-Measure'] == maxFMeasure["max(F-Measure)"])
    .select("threshold").head()["threshold"])

fMeasure.where(fMeasure['F-Measure'] == maxFMeasure["max(F-Measure)"]).show()
# 重新设置最佳的切点
lr.setThreshold(bestThreshold)
model2 = lr.fit(training)
trainingSummary2 = model2.summary
trainingSummary2.areaUnderROC
spark.stop()
